########################################################################
#  Pyfm - personal finance data-base
#  Copyright (C) 2007,8,9 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
"""
You will need to add pdftotext.exe to your PATH. It can be extracted from ZIP
file xpdf...-win32.zip downloaded from http://www.foolabs.com/xpdf/download.html
"""
help="""
"""
import os
import subprocess
import logging
from scanner import *
from csv2flt import *
from n3tagvalue import *

def pre(fname,tmpname,hashing=False):
# stdout returns empty with
#    popen= subprocess.Popen(['pdftotext','-layout', fname, 'con'], # con - redirect to output
#                         stderr=subprocess.PIPE,stdout=tmpname)
#    (stdoutdata, stderrdata) = popen.communicate()
#    rc = popen.returncode:
    tmpfile=fname+'.txt'
    rc= subprocess.call(['pdftotext','-layout',fname,tmpfile])
    if rc:
        raise Error('converting %s to temp file gave %d'%(fname,rc))
    fp=open(tmpfile,'r')
    for line in fp:
        tmpname.write(line)
    fp.close()
    os.remove(tmpfile)
    
####################
# scanner templates
# put in 'l' all the lines starting with "Nominal/Number"
t_remove_header="""*@'(?!\s*Statement of Investments as of).*'@
@'\s*Statement of Investments as of\s+(?P<m>\d\d)\.(?P<d>\d\d)\.(?P<y>\d{4}).*'@
@'.*'@
@'\s*Portfolio Number\s+(?P<acctid>[\d\-]+).*'@
*@'(?!Nominal/Number).*'@
*@'(?P<l>.*)'@"""

# put in 'l' all lines until reaching "Nominal/Number...cash flow"
t_remove_tail="""*@'(?P<l>(?!Nominal/Number\s+Description\s+Market price\s+Net purchase value\s+Market value\s+P/L security / %\s+Exp. cash flow).*)'@
*@'.*'@"""

#Each asset is made out of 3 lines
t_get_asset="""*@'(?!(([A-Z]{3})?\s{1,20}(((\-?\d{1,3}(,\d{3})*(\.\d\d)?)|0))\s{1,30})(\(?\w+|(\d+|\d*\.\d+)\%\s)).*'@
@'(?P<l>.*)'@
@'(?P<l>.*)'@
@'(?P<l>.*)'@
*@'(?P<n>.*)'@"""

###############################
# regular expressions
# first and second lines of each asset should look like this:
re_asset_1st_line =re.compile("""^
(?P<CUR>[A-Z]{3})?\s{1,20}
(?P<UNITS>((\-?\d{1,3}(,\d{3})*(\.\d\d)?)|0))\s{1,30}
(?P<SECNAME>[^\s]+(\s[^\s]+)*)\s{4,}
(((?P<SECID>\w{12})
 (\s+(?P<YIELD>(\d*\.(\d{2,})|NA))\%?)?\s+
 (?P<CUR1>[A-Z]{3})\s+
 (?P<UNITPRICE>\d+\.\d{4})\s
 (?P<m>\d\d)\.(?P<d>\d\d)\.(?P<y>\d\d)\s{10,26}
)|(\s{60,}))
(?P<MKTVAL>\d{1,3}(,\d{3})*)\s{6,18}\d+\.\d\d$""",re.X)
re_asset_2nd_line=re.compile("""^
((\s{20,30}
  (?P<SECNAME>[^\s]+(\s[^\s]+)*)(\s{4,}.*)?
 )|)$""",re.X)
re_asset_3nd_line=re.compile("""^
((\s{20,30}
  Sector:\s+(?P<SECTOR>[^\s]+(\s+[^\s\d][^\s]*)*)\s*((?P<m>\d\d)\.(?P<d>\d\d)\.(?P<y>\d\d))?.*
 )|)$""",re.X)

def flat(finp,fout,context=None,hashing=False):
    n3header(fout,"$Id$"[1:-1],context)
    namespace["commanumre"]=commanumre
    namespace["commaintre"]=commaintre
    namespace["eurdre"]=eurdre
    try:
        rows=scanner(t_remove_header,finp)
        txt=[]
        month=''
        day=''
        year=''
        acctid=''
        for r in rows:
            if 'm' in r: month=r['m']
            if 'd' in r: day=r['d']
            if 'y' in r: year=r['y']
            if 'acctid' in r: acctid=r['acctid']
            if 'l' in r: txt.append(r['l'])
        txt='\n'.join(txt)
        rows=scanner(t_remove_tail,txt)
        txt=[]
        for r in rows:
            txt.append(r['l'])
        txt='\n'.join(txt)

        dtasof=year+month+day
        while txt:
            secname=cur=units=secid=secidtype=price=mktval=None
            line_acctid=acctid
            line_dtasof=None
            rows=scanner(t_get_asset,txt)
            ctxt=[]
            ntxt=[]
            for r in rows:
                if 'l' in r: ctxt.append(r['l'])
                if 'n' in r: ntxt.append(r['n'])
            m=re_asset_1st_line.match(ctxt[0])
            if not m:
                raise Exception('bad format 1')
            mdict=m.groupdict()
            m=re_asset_2nd_line.match(ctxt[1])
            if not m:
                raise Exception('bad format 2')
            mdict2=m.groupdict()
            txt='\n'.join(ntxt)
            cur=mdict['CUR']
            if not cur:
                cur=mdict['CUR1']
            units=mdict['UNITS']
            units=units.replace(',','')
            secname=mdict['SECNAME']
            if mdict2['SECNAME'] and not mdict2['SECNAME'].startswith('Sector: '):
                secname+=' '+mdict2['SECNAME']
            if mdict['SECID']:
                secid=mdict['SECID']
                secidtype='ISIN'
            else:
                if secname.startswith('Current Account '):
                    if not cur:
                        print "Missing Currency"
                    secid=cur
                    secidtype='CHECKING'
                    m=re.match('^Current Account (?P<ACCTID>\d{4}\-\d{6}\-\w+(\-\w+)?)$',secname)
                    if not m:
                        print "Bad Current Account", secname
                    else:
                        line_acctid=m.group('ACCTID')
                    secname=None
                else:
                    secidtype='MNYMARKET'
                    secid=cur
                    m=re.match('^(\d+(\.\d+)?|\.\d+)% \-(?P<ACCTID>\d{6}\-\w+(\-\w+)?) \d{2}\.\d{2}\.\d{2}Overnight Money Fiduciary$',secname)
                    if not m:
                        print "Bad money market",secname
                    else:
                        line_acctid=m.group('ACCTID')
                    secname=None
            price = mdict['UNITPRICE'] if mdict['UNITPRICE'] else "1"
            yld = None
            if mdict['YIELD']:
                price = float(price) / 100.
                yld=mdict['YIELD']
            if mdict['MKTVAL']:
                mktval=mdict['MKTVAL'].replace(',','')
            if mdict['m']:
                line_dtasof='20'+mdict['y']+mdict['m']+mdict['d']
            if mdict2['SECNAME'] and mdict2['SECNAME'].startswith('Sector: '):
                l3=ctxt[1]
            else:
                l3=ctxt[2]
            m=re_asset_3nd_line.match(l3)
            if not m:
                raise Exception('bad format 3')
            mdict3=m.groupdict()
            fiassetclass=mdict3['SECTOR']
            if mdict3['m']:
                dtmat='20'+mdict3['y']+mdict3['m']+mdict3['d']
            else:
                dtmat=None

            print >>fout,"a3:flatLine [ ",
            if secname: print >>fout,stagvalue('SECNAME',secname),				
            print >>fout,stagvalue('CURRENCY',cur),
            print >>fout,stagvalue('UNITS',units),
            print >>fout,stagvalue('ACCTID',line_acctid,hashing=hashing),
            print >>fout,n3secid(secidtype,secid),
            if price and price!="NA": print >>fout,stagvalue('UNITPRICE',price),
            if line_dtasof: print >>fout,stagvalue('dtasof',line_dtasof),
            print >>fout,stagvalue('DTASOF',dtasof),
            if yld and yld!="NA": print >>fout,stagvalue('YIELD',yld),
            if mktval: print >>fout,stagvalue('MKTVAL',mktval),
            if fiassetclass: print >>fout,stagvalue('FIASSETCLASS',fiassetclass),
            if dtmat: print >>fout,stagvalue('DTMAT',dtmat),
            print >>fout,"];"
    except scannerror, v:
        pass
    finally:
        print >>fout,"."
        finp.close()
